package com.etoak.crawl.main;

import com.etoak.crawl.link.LinkFilter;
import com.etoak.crawl.link.Links;
import com.etoak.crawl.page.Page;
import com.etoak.crawl.page.PageParserTool;
import com.etoak.crawl.page.RequestAndResponseTool;
import com.etoak.crawl.pojo.Product;
import jxl.Workbook;
import jxl.write.Label;
import jxl.write.WritableSheet;
import jxl.write.WritableWorkbook;
import org.apache.poi.hssf.usermodel.*;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.xml.ws.Service;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;

public class MyCrawler {

    /**
     * 使用种子初始化 URL 队列
     *
     * @param seeds 种子 URL
     * @return
     */
    private void initCrawlerWithSeeds(String[] seeds) {
        for (int i = 0; i < seeds.length; i++) {
            Links.addUnvisitedUrlQueue(seeds[i]);
        }
    }




    int i = 0;

    /**
     * 抓取过程
     *
     * @param seeds
     * @return
     */
    public List<Product> crawlingpro(String[] seeds, String title) throws IOException {

        List<Product> prolist = new ArrayList<Product>();
        //初始化 URL 队列
        initCrawlerWithSeeds(seeds);

        //定义过滤器，提取以 http://www.baidu.com 开头的链接
        LinkFilter filter = new LinkFilter() {
            public boolean accept(String url) {
                if (url.startsWith("http://www.bzjw.com"))
                    return true;
                else
                    return false;
            }
        };

        //循环条件：待抓取的链接不空且抓取的网页不多于 1000
        while (!Links.unVisitedUrlQueueIsEmpty() && Links.getVisitedUrlNum() <= 1000) {

            //先从待访问的序列中取出第一个；
            String visitUrl = (String) Links.removeHeadOfUnVisitedUrlQueue();
            if (visitUrl == null) {
                continue;
            }

            //根据URL得到page;
            Page page = RequestAndResponseTool.sendRequstAndGetResponse(visitUrl);

            //产品实体类
            Product pro = new Product();


            //对page进行处理： 访问DOM的某个标签
            Elements es = PageParserTool.select(page, "div.detail");


            //span标签 型号
            String itemsen = es.select("span").text();
            System.err.println("型号:" + itemsen);
            pro.setModel(itemsen);

            //中文名称
            Elements itemsench = es.select("h2");
            String[] newitemsench = itemsench.toString().split(">");
            String[] newitemsenchs = newitemsench[3].split("<");
            System.err.println("中文名称:" + newitemsenchs[0]);
            pro.setChname(newitemsenchs[0]);

            //
            String items = es.select("p").text();
            //System.err.println("p:"+itemsenp);

            //代替
            String[] itemsp = items.split("）");
            String[] itemsps = itemsp[0].split("（");
            System.err.println("代替:" + itemsps[1]);
            pro.setReplace(itemsps[1]);
            System.err.println("英文名称:" + itemsp[1]);
            pro.setEnname(newitemsenchs[0]);

            //图片路径

            Elements pngs = es.select("img[src$=.jpg]");
            for (Element link : pngs) {
                String linkHref = link.attr("src");
                System.err.println("图片路径:" + linkHref);
                pro.setPciurl(linkHref);
                dow(linkHref);
            }

            System.err.println("二级分类:" + title);
            pro.setTwotitle(title);
            //id
            i++;
            pro.setId(i);

            prolist.add(pro);


            //将保存文件
            //FileTool.saveToLocal(page);

            //将已经访问过的链接放入已访问的链接中；
            Links.addVisitedUrlSet(visitUrl);

            //得到超链接
            //Set<String> links = PageParserTool.getLinks(page, "img");
            //List<String> links = PageParserTool.arbitrarily(page,"div.l");
            /*for (String link : links) {
                Links.addUnvisitedUrlQueue(link);
                System.err.println("新增爬取路径: " + link);
            }*/
        }
        for (Product prolists : prolist) {
            System.err.println(prolists.toString());
        }

        return prolist;

    }


    public static void downexcel(List<Product> list) {

        try {
            WritableWorkbook wwb = null;

            // 查询数据库中所有的数据
            // List<Currency> list= currencyService.findall(0,1000000000);



            // 创建可写入的Excel工作簿
            File file = new File("E:\\1.xls");
            if (!file.exists()) {
                file.createNewFile();
            }
            // 以fileName为文件名来创建一个Workbook
            wwb = Workbook.createWorkbook(file);

            // 创建工作表
            WritableSheet ws = wwb.createSheet("Test Shee 1", 0);

            // 要插入到的Excel表格的行号，默认从0开始
            Label labelId = new Label(0, 0, "编号");// 表示第
            Label labeltitle = new Label(1, 0, "型号");
            Label labelcodes = new Label(2, 0, "中文名称");
            Label labelspecifications = new Label(3, 0, "代替");
            Label labelonename = new Label(4, 0, "英文名称");
            Label labeltwoname = new Label(5, 0, "图片路径");
            Label labelthreename = new Label(6, 0, "二级分类名称");


            ws.addCell(labelId);
            ws.addCell(labeltitle);
            ws.addCell(labelcodes);
            ws.addCell(labelspecifications);
            ws.addCell(labelonename);
            ws.addCell(labeltwoname);
            ws.addCell(labelthreename);

            for (int i = 0; i < list.size(); i++) {

                Label labelId_i = new Label(0, i + 1, list.get(i).getId() + "");
                Label labeltitle_i = new Label(1, i + 1, list.get(i).getModel());
                Label labelcodes_i = new Label(2, i + 1, list.get(i).getChname());
                Label labelspecifications_i = new Label(3, i + 1, list.get(i).getReplace());
                Label labelonename_i = new Label(4, i + 1, list.get(i).getEnname());
                Label labeltwoname_i = new Label(5, i + 1, list.get(i).getPciurl());
                Label labelthreename_i = new Label(6, i + 1, list.get(i).getTwotitle());
                ws.addCell(labelId_i);
                ws.addCell(labeltitle_i);
                ws.addCell(labelcodes_i);
                ws.addCell(labelspecifications_i);
                ws.addCell(labelonename_i);
                ws.addCell(labeltwoname_i);
                ws.addCell(labelthreename_i);

            }

            // 写进文档
            wwb.write();
            // 关闭Excel工作簿对象
            wwb.close();


        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }


    /**
     * 下载文件
     *
     * @param urls
     * @throws IOException
     */
    public void dow(String urls) throws IOException {
        URL url = new URL(urls);
        URLConnection con = url.openConnection();
        String[] newurl = urls.split("/");
        FileOutputStream out = new FileOutputStream("E:/tu/" + newurl[4]);
        InputStream ins = con.getInputStream();
        byte[] b = new byte[1024];
        int i = 0;
        while ((i = ins.read(b)) != -1) {
            out.write(b, 0, i);
        }
        ins.close();
        out.close();
    }


    //main 方法入口
    public static void main(String[] args) throws IOException {
        MyCrawlerlist crawler = new MyCrawlerlist();

        List<Product> prolist  =crawler.crawling(new String[]{"http://www.bzjw.com/standard/index.asp?class_id=526"});
        downexcel(prolist);
    }
}
